In [350]:
import logging
from pathlib import Path

from matplotlib import rc
from IPython.display import display

from pySHOC import shocCampaign, shocHDU

# ensure we return shocCampaign when slicing
shocCampaign.set_returned_type(None)   # FIXME! remove the need for this

# setup logging
rootlog = logging.getLogger()
rootlog.setLevel(logging.INFO)
In [351]:
# Load data
root_folder = Path('/media/Oceanus/UCT/Observing/data/sources/J1928-5001')
fig_folder = Path('/home/hannes/Documents/papers/dev/J1928/figures')
rc('savefig', directory=fig_folder)

run = shocCampaign.load_dir(root_folder / 'SHOC/raw')
In [352]:
# Some of the frames are labelled incorrectly.
# OBSTYPE discovery + grouping
g, idx = run.group_by(shocHDU.guess_obstype, return_index=True)  
# obstype = run.guess_obstype(plot=True)  # to display stats for obstype decision
tables = g.pprint();  
# notice `shocHDU.guess_obstype` identifies frames correctly from the 
# distribution of pixel values, despite incorrect header info!
                                                                                                                                                           
⎪                                                                     shocObsGroups                                                                     ⎪  
⎪instrument = SHOC 1                                                                                                                                    ⎪  
⎪filters.B  = ∅                                                                                                                                         ⎪  
⎪                                                      ⎪f…⎪                          ⎪      readout       ⎪                   timing                    ⎪  
⎪#  ⎪     filename     ⎪ tel ⎪     target     ⎪obstype ⎪A ⎪   n   ⎪  ishape   ⎪ bin  ⎪γₚᵣₑ ⎪     mode     ⎪          t0          ⎪  tExp  ⎪  duration   ⎪  
⎪0  ⎪SHA_20150715.0002 ⎪74in ⎪SKYFLAT         ⎪flat    ⎪∅ ⎪     15⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-07-15 16:07:32.0 ⎪ 0.14642⎪  00ʰ00ᵐ02.3ˢ⎪ ⎫ 
⎪1  ⎪20130616.0002     ⎪None ⎪None            ⎪None    ⎪∅ ⎪     50⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz CON     ⎪2013-06-16 16:02:16.9 ⎪ 0.14642⎪  00ʰ00ᵐ07.7ˢ⎪ ⎪ 
⎪2  ⎪20130617.0001     ⎪None ⎪None            ⎪None    ⎪∅ ⎪    100⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz CON     ⎪2013-06-17 16:02:29.9 ⎪ 0.14642⎪  00ʰ00ᵐ15.3ˢ⎪ ⎪ 
⎪3  ⎪202130615.0002    ⎪None ⎪None            ⎪None    ⎪∅ ⎪     50⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz CON     ⎪2013-06-15 16:05:53.9 ⎪ 0.14642⎪  00ʰ00ᵐ07.7ˢ⎪ ⎪ 
⎪4  ⎪SHA_20150606.0008 ⎪74in ⎪                ⎪        ⎪∅ ⎪     10⎪(128, 128) ⎪8x8   ⎪  1.0⎪3 MHz CON     ⎪2015-06-06 16:20:40.0 ⎪    10.0⎪  00ʰ01ᵐ40.1ˢ⎪ ⎪ 
⎪5  ⎪SHA_20150607.0003 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪     30⎪(128, 128) ⎪8x8   ⎪  1.0⎪3 MHz CON     ⎪2015-06-07 15:57:58.0 ⎪  0.0558⎪  00ʰ00ᵐ01.9ˢ⎪ ⎬ flat 
⎪6  ⎪SHA_20150904.0012 ⎪74in ⎪SKYFLAT         ⎪flat    ⎪I ⎪     15⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-09-04 16:34:43.0 ⎪ 0.14642⎪  00ʰ00ᵐ02.3ˢ⎪ ⎪ 
⎪7  ⎪SHA_20150904.0022 ⎪74in ⎪SKYFLAT         ⎪flat    ⎪∅ ⎪     15⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-09-04 16:39:23.0 ⎪ 0.14642⎪  00ʰ00ᵐ02.3ˢ⎪ ⎪ 
⎪8  ⎪SHA_20160706.0004 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪    405⎪(85, 85)   ⎪12x12 ⎪  2.4⎪1 MHz CON     ⎪2016-07-06 16:14:41.0 ⎪     0.3⎪  00ʰ02ᵐ04.2ˢ⎪ ⎪ 
⎪9  ⎪SHA_20160707.0004 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪    633⎪(85, 85)   ⎪12x12 ⎪  1.0⎪1 MHz CON     ⎪2016-07-07 16:11:08.0 ⎪ 0.11847⎪  00ʰ01ᵐ19.3ˢ⎪ ⎪ 
⎪10 ⎪SHA_20160710.0003 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪    500⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2016-07-10 16:06:48.0 ⎪ 0.14642⎪  00ʰ01ᵐ16.6ˢ⎪ ⎪ 
⎪11 ⎪SHA_20160710.0005 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪    500⎪(85, 85)   ⎪12x12 ⎪  1.0⎪1 MHz CON     ⎪2016-07-10 16:10:53.0 ⎪ 0.11847⎪  00ʰ01ᵐ02.6ˢ⎪⎪   ⎪                  ⎪     ⎪                ⎪        ⎪  ⎪   2323⎪           ⎪      ⎪     ⎪              ⎪                      ⎪        ⎪  00ʰ08ᵐ02.2ˢ⎪  
⎪0  ⎪20130616.0030     ⎪None ⎪None            ⎪None    ⎪∅ ⎪  35000⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-16 21:45:55.9 ⎪     0.1⎪ 01ʰ02ᵐ16.6ˢ ⎪ ⎫ 
⎪1  ⎪20130616.0031     ⎪None ⎪None            ⎪None    ⎪∅ ⎪  35000⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-16 22:48:41.9 ⎪     0.1⎪ 01ʰ02ᵐ16.6ˢ ⎪ ⎪ 
⎪2  ⎪20130617.0030     ⎪None ⎪None            ⎪None    ⎪∅ ⎪  35000⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-17 21:49:58.9 ⎪     0.1⎪ 01ʰ02ᵐ16.6ˢ ⎪ ⎪ 
⎪3  ⎪20130617.0031     ⎪None ⎪None            ⎪None    ⎪∅ ⎪  35000⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-17 22:52:57.9 ⎪     0.1⎪ 01ʰ02ᵐ16.6ˢ ⎪ ⎪ 
⎪4  ⎪20130618.0030     ⎪None ⎪None            ⎪None    ⎪∅ ⎪  35000⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 125 ⎪2013-06-18 22:41:04.9 ⎪     0.1⎪ 01ʰ02ᵐ16.6ˢ ⎪ ⎪ 
⎪5  ⎪20130618.0031     ⎪None ⎪None            ⎪None    ⎪∅ ⎪  22999⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 125 ⎪2013-06-18 23:43:58.9 ⎪     0.1⎪ 00ʰ40ᵐ55.4ˢ ⎪ ⎪ 
⎪6  ⎪202130615.0030    ⎪None ⎪None            ⎪None    ⎪∅ ⎪  20000⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz EM: 50  ⎪2013-06-15 23:15:00.9 ⎪ 0.14862⎪ 00ʰ51ᵐ47.6ˢ ⎪ ⎪ 
⎪7  ⎪202130615.0031    ⎪None ⎪None            ⎪None    ⎪∅ ⎪  20000⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz EM: 50  ⎪2013-06-16 00:07:07.9 ⎪ 0.14862⎪ 00ʰ51ᵐ47.6ˢ ⎪ ⎪ 
⎪8  ⎪202130615.0032    ⎪None ⎪None            ⎪None    ⎪∅ ⎪  10000⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz EM: 50  ⎪2013-06-16 01:01:08.9 ⎪ 0.14862⎪ 00ʰ25ᵐ53.8ˢ ⎪ ⎪ 
⎪9  ⎪SHA_20150606.0300 ⎪74in ⎪CTCVJ1928-5001  ⎪object  ⎪∅ ⎪   4824⎪(128, 128) ⎪8x8   ⎪  1.0⎪3 MHz CON     ⎪2015-06-07 01:25:01.0 ⎪ 0.49324⎪ 00ʰ40ᵐ12.0ˢ*⎪ ⎬ object 
⎪10 ⎪SHA_20150715.0200 ⎪74in ⎪CXOUJ1723-3734  ⎪object  ⎪∅ ⎪   3584⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-07-16 01:43:03.0 ⎪ 0.99324⎪ 00ʰ59ᵐ44.0ˢ*⎪ ⎪ 
⎪11 ⎪SHA_20150904.0201 ⎪74in ⎪A 48            ⎪object  ⎪I ⎪   7201⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-09-04 19:35:00.0 ⎪ 0.99324⎪ 02ʰ00ᵐ01.0ˢ*⎪ ⎪ 
⎪12 ⎪SHA_20150904.0202 ⎪74in ⎪A 48            ⎪object  ⎪∅ ⎪   9001⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-09-04 22:40:00.0 ⎪ 0.99324⎪ 02ʰ30ᵐ01.0ˢ*⎪ ⎪ 
⎪13 ⎪SHA_20150905.0101 ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪  12600⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-09-05 17:43:00.0 ⎪ 0.99324⎪ 03ʰ30ᵐ00.0ˢ*⎪ ⎪ 
⎪14 ⎪SHA_20160706.0007 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪  18253⎪(85, 85)   ⎪12x12 ⎪  4.9⎪1 MHz EM: 50  ⎪2016-07-06 23:23:08.0 ⎪     0.5⎪ 02ʰ34ᵐ09.9ˢ ⎪ ⎪ 
⎪15 ⎪SHA_20160707.0030 ⎪74in ⎪                ⎪object  ⎪∅ ⎪   4301⎪(85, 85)   ⎪12x12 ⎪  4.9⎪1 MHz EM: 50  ⎪2016-07-07 22:18:55.0 ⎪     0.5⎪ 00ʰ36ᵐ19.6ˢ ⎪ ⎪ 
⎪16 ⎪SHA_20160707.0031 ⎪74in ⎪                ⎪object  ⎪∅ ⎪  28000⎪(85, 85)   ⎪12x12 ⎪  4.9⎪1 MHz EM: 50  ⎪2016-07-07 22:55:21.0 ⎪    0.25⎪ 01ʰ59ᵐ49.3ˢ ⎪ ⎪ 
⎪17 ⎪SHA_20160711.0001 ⎪74in ⎪                ⎪object  ⎪∅ ⎪  37505⎪(128, 128) ⎪8x8   ⎪  4.9⎪1 MHz EM: 65  ⎪2016-07-11 22:51:50.0 ⎪    0.25⎪ 02ʰ40ᵐ29.8ˢ ⎪ ⎪ 
⎪18 ⎪SHA_20171002.0010 ⎪40in ⎪J1928           ⎪object  ⎪∅ ⎪    670⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz CON     ⎪2017-10-02 18:35:38.0 ⎪     0.5⎪ 00ʰ05ᵐ39.5ˢ ⎪ ⎪ 
⎪19 ⎪SHA_20171002.0011 ⎪40in ⎪J1928           ⎪object  ⎪∅ ⎪  12498⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz CON     ⎪2017-10-02 18:59:44.0 ⎪     0.5⎪ 01ʰ45ᵐ33.5ˢ ⎪⎪   ⎪                  ⎪     ⎪                ⎪        ⎪  ⎪ 386436⎪           ⎪      ⎪     ⎪              ⎪                      ⎪        ⎪  27ʰ23ᵐ46.9ˢ⎪  
⎪0  ⎪20130616.0032     ⎪None ⎪None            ⎪None    ⎪∅ ⎪      1⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 5   ⎪2013-06-17 00:08:16.9 ⎪     0.1⎪ 00ʰ00ᵐ00.1ˢ ⎪ ⎫ 
⎪1  ⎪20130617.0032     ⎪None ⎪None            ⎪None    ⎪∅ ⎪      1⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz CON     ⎪2013-06-18 00:04:42.9 ⎪     0.1⎪ 00ʰ00ᵐ00.1ˢ ⎪ ⎬ bad 
⎪2  ⎪SHA_20150904.0203 ⎪74in ⎪A 48            ⎪object  ⎪∅ ⎪      1⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪1970-01-01 00:00:00.0 ⎪ 0.99324⎪ 00ʰ00ᵐ01.0ˢ*⎪ ⎪ 
⎪3  ⎪SHA_20150904.0204 ⎪74in ⎪A 48            ⎪object  ⎪∅ ⎪      1⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪1970-01-01 00:00:00.0 ⎪ 0.99324⎪ 00ʰ00ᵐ01.0ˢ*⎪⎪   ⎪                  ⎪     ⎪                ⎪        ⎪  ⎪      4⎪           ⎪      ⎪     ⎪              ⎪                      ⎪        ⎪  00ʰ00ᵐ02.2ˢ⎪  
⎪0  ⎪20130618.0104     ⎪None ⎪None            ⎪None    ⎪∅ ⎪    100⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-19 02:13:32.0 ⎪  0.0566⎪  00ʰ00ᵐ06.3ˢ⎪ ⎫ 
⎪1  ⎪20130618.0105     ⎪None ⎪None            ⎪None    ⎪∅ ⎪    100⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 125 ⎪2013-06-19 02:14:01.0 ⎪  0.0566⎪  00ʰ00ᵐ06.3ˢ⎪ ⎪ 
⎪2  ⎪202130615.0101    ⎪None ⎪None            ⎪None    ⎪∅ ⎪     50⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz EM: 50  ⎪2013-06-16 04:29:56.9 ⎪ 0.14862⎪  00ʰ00ᵐ07.8ˢ⎪ ⎪ 
⎪3  ⎪SHA_20150607.0010 ⎪74in ⎪                ⎪bias    ⎪∅ ⎪     30⎪(128, 128) ⎪8x8   ⎪  1.0⎪3 MHz CON     ⎪2015-06-07 16:01:12.0 ⎪  0.0558⎪  00ʰ00ᵐ01.9ˢ⎪ ⎪ 
⎪4  ⎪SHA_20150607.0011 ⎪74in ⎪                ⎪bias    ⎪∅ ⎪     30⎪(128, 128) ⎪8x8   ⎪  1.0⎪3 MHz CON     ⎪2015-06-07 16:01:21.0 ⎪  0.0558⎪  00ʰ00ᵐ01.9ˢ⎪ ⎪ 
⎪5  ⎪SHA_20150607.0012 ⎪74in ⎪                ⎪bias    ⎪∅ ⎪     30⎪(128, 128) ⎪8x8   ⎪  1.0⎪3 MHz CON     ⎪2015-06-07 16:01:38.0 ⎪  0.0558⎪  00ʰ00ᵐ01.9ˢ⎪ ⎪ 
⎪6  ⎪SHA_20150715.0051 ⎪74in ⎪BIAS            ⎪bias    ⎪∅ ⎪     15⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-07-16 03:00:02.0 ⎪ 0.14642⎪  00ʰ00ᵐ02.3ˢ⎪ ⎬ dark 
⎪7  ⎪SHA_20150904.0002 ⎪74in ⎪BIAS            ⎪bias    ⎪I ⎪     15⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-09-04 16:01:18.0 ⎪ 0.14642⎪  00ʰ00ᵐ02.3ˢ⎪ ⎪ 
⎪8  ⎪SHA_20160712.0105 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪   1000⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz CON     ⎪2016-07-13 05:33:59.0 ⎪ 0.14642⎪  00ʰ02ᵐ33.2ˢ⎪ ⎪ 
⎪9  ⎪SHA_20160712.0109 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪   1000⎪(128, 128) ⎪8x8   ⎪  4.9⎪1 MHz EM: 65  ⎪2016-07-13 05:46:45.0 ⎪ 0.14862⎪  00ʰ02ᵐ35.4ˢ⎪ ⎪ 
⎪10 ⎪SHA_20160712.0114 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪   1000⎪(102, 102) ⎪10x10 ⎪  4.9⎪1 MHz EM: 50  ⎪2016-07-13 06:00:42.0 ⎪ 0.12067⎪  00ʰ02ᵐ07.4ˢ⎪ ⎪ 
⎪11 ⎪SHA_20160712.0115 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪   1000⎪(85, 85)   ⎪12x12 ⎪  1.0⎪1 MHz CON     ⎪2016-07-13 06:04:13.0 ⎪  0.1002⎪  00ʰ01ᵐ47.0ˢ⎪ ⎪ 
⎪12 ⎪SHA_20160712.0116 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪   1000⎪(85, 85)   ⎪12x12 ⎪  2.4⎪1 MHz CON     ⎪2016-07-13 06:06:08.0 ⎪  0.1002⎪  00ʰ01ᵐ47.0ˢ⎪ ⎪ 
⎪13 ⎪SHA_20160712.0117 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪   1000⎪(85, 85)   ⎪12x12 ⎪  4.9⎪1 MHz EM: 50  ⎪2016-07-13 06:08:52.0 ⎪ 0.10242⎪  00ʰ01ᵐ49.2ˢ⎪⎪   ⎪                  ⎪     ⎪                ⎪        ⎪  ⎪   6370⎪           ⎪      ⎪     ⎪              ⎪                      ⎪        ⎪  00ʰ13ᵐ09.8ˢ⎪  
* tExp: timing.trigger.is_gps_loop  
* t0: timing.trigger.is_gps  
In [353]:
# Apply the guessed observation type labels (flat / dark / bad)
# Note this updates the attributes of the shocHDU's, but leaves the fits 
# headers untouched
for obstype, r in g.items():
    # set attributes on HDU objects - not yet in header
    r.set_attrs(obstype=obstype)

# remove bad files
g.pop('bad', None)

# set the target name
run_src = g['object']
run_src.set_attrs(target='CTCV J1928-5001')

# add telescope info for old data.  We will need this later
for obstype in ['object', 'flat']:
    run = g[obstype]
    is74in = np.equal(run.attrs('telescope'), None)
    run[is74in].set_attrs(telescope='74in')

# print target observations
run_src.sort_by('date').pprint();   # todo: list by date; export to latex
                                                                                                                                
⎪                                                         shocCampaign                                                         ⎪
⎪camera  = SHOC 1                                                                                                              ⎪
⎪target  = CTCV J1928-5001                                                                                                     ⎪
⎪obstype = object                                                                                                              ⎪
⎪B       = ∅                                                                                                                   ⎪
⎪                            ⎪f…⎪                          ⎪      readout       ⎪                    timing                    ⎪
⎪#  ⎪     filename     ⎪ tel ⎪A ⎪   n   ⎪  ishape   ⎪ bin  ⎪γₚᵣₑ ⎪     mode     ⎪          t0           ⎪  tExp   ⎪  duration  ⎪
⎪0  ⎪202130615.0030    ⎪74in ⎪∅ ⎪  20000⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz EM: 50  ⎪2013-06-15 23:15:00.9  ⎪ 0.14862 ⎪ 00ʰ51ᵐ47.6ˢ⎪
⎪1  ⎪20130616.0030     ⎪74in ⎪∅ ⎪  35000⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-16 21:45:55.9  ⎪     0.1 ⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪2  ⎪20130616.0031     ⎪74in ⎪∅ ⎪  35000⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-16 22:48:41.9  ⎪     0.1 ⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪3  ⎪202130615.0031    ⎪74in ⎪∅ ⎪  20000⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz EM: 50  ⎪2013-06-16 00:07:07.9  ⎪ 0.14862 ⎪ 00ʰ51ᵐ47.6ˢ⎪
⎪4  ⎪202130615.0032    ⎪74in ⎪∅ ⎪  10000⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz EM: 50  ⎪2013-06-16 01:01:08.9  ⎪ 0.14862 ⎪ 00ʰ25ᵐ53.8ˢ⎪
⎪5  ⎪20130617.0030     ⎪74in ⎪∅ ⎪  35000⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-17 21:49:58.9  ⎪     0.1 ⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪6  ⎪20130617.0031     ⎪74in ⎪∅ ⎪  35000⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-17 22:52:57.9  ⎪     0.1 ⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪7  ⎪20130618.0030     ⎪74in ⎪∅ ⎪  35000⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 125 ⎪2013-06-18 22:41:04.9  ⎪     0.1 ⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪8  ⎪20130618.0031     ⎪74in ⎪∅ ⎪  22999⎪(128, 128) ⎪8x8   ⎪  2.4⎪3 MHz EM: 125 ⎪2013-06-18 23:43:58.9  ⎪     0.1 ⎪ 00ʰ40ᵐ55.4ˢ⎪
⎪9  ⎪SHA_20150606.0300 ⎪74in ⎪∅ ⎪   4824⎪(128, 128) ⎪8x8   ⎪  1.0⎪3 MHz CON     ⎪2015-06-07 01:25:01.0* ⎪ 0.49324*⎪ 00ʰ40ᵐ12.0ˢ⎪
⎪10 ⎪SHA_20150715.0200 ⎪74in ⎪∅ ⎪   3584⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-07-16 01:43:03.0* ⎪ 0.99324*⎪ 00ʰ59ᵐ44.0ˢ⎪
⎪11 ⎪SHA_20150904.0201 ⎪74in ⎪I ⎪   7201⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-09-04 19:35:00.0* ⎪ 0.99324*⎪ 02ʰ00ᵐ01.0ˢ⎪
⎪12 ⎪SHA_20150904.0202 ⎪74in ⎪∅ ⎪   9001⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-09-04 22:40:00.0* ⎪ 0.99324*⎪ 02ʰ30ᵐ01.0ˢ⎪
⎪13 ⎪SHA_20150905.0101 ⎪74in ⎪∅ ⎪  12600⎪(128, 128) ⎪8x8   ⎪  1.0⎪1 MHz CON     ⎪2015-09-05 17:43:00.0* ⎪ 0.99324*⎪ 03ʰ30ᵐ00.0ˢ⎪
⎪14 ⎪SHA_20160706.0007 ⎪74in ⎪∅ ⎪  18253⎪(85, 85)   ⎪12x12 ⎪  4.9⎪1 MHz EM: 50  ⎪2016-07-06 23:23:08.0  ⎪     0.5 ⎪ 02ʰ34ᵐ09.9ˢ⎪
⎪15 ⎪SHA_20160707.0030 ⎪74in ⎪∅ ⎪   4301⎪(85, 85)   ⎪12x12 ⎪  4.9⎪1 MHz EM: 50  ⎪2016-07-07 22:18:55.0  ⎪     0.5 ⎪ 00ʰ36ᵐ19.6ˢ⎪
⎪16 ⎪SHA_20160707.0031 ⎪74in ⎪∅ ⎪  28000⎪(85, 85)   ⎪12x12 ⎪  4.9⎪1 MHz EM: 50  ⎪2016-07-07 22:55:21.0  ⎪    0.25 ⎪ 01ʰ59ᵐ49.3ˢ⎪
⎪17 ⎪SHA_20160711.0001 ⎪74in ⎪∅ ⎪  37505⎪(128, 128) ⎪8x8   ⎪  4.9⎪1 MHz EM: 65  ⎪2016-07-11 22:51:50.0  ⎪    0.25 ⎪ 02ʰ40ᵐ29.8ˢ⎪
⎪18 ⎪SHA_20171002.0010 ⎪40in ⎪∅ ⎪    670⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz CON     ⎪2017-10-02 18:35:38.0  ⎪     0.5 ⎪ 00ʰ05ᵐ39.5ˢ⎪
⎪19 ⎪SHA_20171002.0011 ⎪40in ⎪∅ ⎪  12498⎪(128, 128) ⎪8x8   ⎪  2.4⎪1 MHz CON     ⎪2017-10-02 18:59:44.0  ⎪     0.5 ⎪ 01ʰ45ᵐ33.5ˢ⎪
⎪   ⎪                  ⎪     ⎪  ⎪ 386436⎪           ⎪      ⎪     ⎪              ⎪                       ⎪         ⎪ 27ʰ23ᵐ46.9ˢ⎪
* t0: timing.trigger.is_gps
* tExp: timing.trigger.is_gps_loop
In [179]:
# Science images thumbnail grid 
fig, axes, _ = run_src.sort_by('date').thumbnails()
fig.set_size_inches(9, 7)
fig
Out[179]:
In [354]:
# Match calibration frames
ATT_EQUAL_DARK = ('instrument', 'binning', # subrect
                  'readout.preAmpGain', 'readout.outAmp.mode', 'readout.frq')  # <-- 'readout'
ATT_CLOSE_DARK = ('readout.outAmp.emGain', 'timing.t_expose')
ATT_DARK = ATT_EQUAL_DARK, ATT_CLOSE_DARK

# DEBIAS
# need to debias flats & science observations
obs = g['flat'].join(g['object']) 
gobs, gdark = obs.match(g['dark'], *ATT_DARK, keep_nulls=False, report=True)

# compute master darks
mbias = gdark.merge_combine(np.median)
                                                                                                                             
⎪Matched Observations                                                                                                       ⎪
⎪exact: ('instrument', 'binning', 'readout.preAmpGain', 'readout.outAmp.mode', 'readout.frq')                               ⎪
⎪closest: ('readout.outAmp.emGain', 'timing.t_expose')                                                                      ⎪
⎪B = ∅                                                                                                                      ⎪
⎪                                                  ⎪f…⎪                  ⎪   readout    ⎪              timing               ⎪
⎪     filename     ⎪ tel ⎪     target     ⎪obstype ⎪A ⎪  n   ⎪  ishape   ⎪     mode     ⎪          t0          ⎪  duration  ⎪
⎪                                                                                                                           ⎪
⎪group 0:                                       12x12; 1.0; CON; 1; 0; 0.1002
⎪SHA_20160712.0115 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪  1000⎪(85, 85)   ⎪1 MHz CON     ⎪2016-07-13 06:04:13.0 ⎪ 00ʰ01ᵐ47.0ˢ⎪
⎪SHA_20160707.0004 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪   633⎪(85, 85)   ⎪1 MHz CON     ⎪2016-07-07 16:11:08.0 ⎪ 00ʰ01ᵐ19.3ˢ⎪
⎪SHA_20160710.0005 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪   500⎪(85, 85)   ⎪1 MHz CON     ⎪2016-07-10 16:10:53.0 ⎪ 00ʰ01ᵐ02.6ˢ⎪
⎪                                                                                                                           ⎪
⎪group 1:                                       8x8; 2.4; EM; 1; 50; 0.14862
⎪202130615.0101    ⎪None ⎪None            ⎪dark    ⎪∅ ⎪    50⎪(128, 128) ⎪1 MHz EM: 50  ⎪2013-06-16 04:29:56.9 ⎪ 00ʰ00ᵐ07.8ˢ⎪
⎪202130615.0030    ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 20000⎪(128, 128) ⎪1 MHz EM: 50  ⎪2013-06-15 23:15:00.9 ⎪ 00ʰ51ᵐ47.6ˢ⎪
⎪202130615.0031    ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 20000⎪(128, 128) ⎪1 MHz EM: 50  ⎪2013-06-16 00:07:07.9 ⎪ 00ʰ51ᵐ47.6ˢ⎪
⎪202130615.0032    ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 10000⎪(128, 128) ⎪1 MHz EM: 50  ⎪2013-06-16 01:01:08.9 ⎪ 00ʰ25ᵐ53.8ˢ⎪
⎪                                                                                                                           ⎪
⎪group 2:                                       8x8; 1.0; CON; 1; 0; 0.14642
⎪SHA_20150715.0051 ⎪74in ⎪BIAS            ⎪dark    ⎪∅ ⎪    15⎪(128, 128) ⎪1 MHz CON     ⎪2015-07-16 03:00:02.0 ⎪ 00ʰ00ᵐ02.3ˢ⎪
⎪SHA_20150904.0002 ⎪74in ⎪BIAS            ⎪dark    ⎪I ⎪    15⎪(128, 128) ⎪1 MHz CON     ⎪2015-09-04 16:01:18.0 ⎪ 00ʰ00ᵐ02.3ˢ⎪
⎪SHA_20150715.0002 ⎪74in ⎪SKYFLAT         ⎪flat    ⎪∅ ⎪    15⎪(128, 128) ⎪1 MHz CON     ⎪2015-07-15 16:07:32.0 ⎪ 00ʰ00ᵐ02.3ˢ⎪
⎪SHA_20150904.0012 ⎪74in ⎪SKYFLAT         ⎪flat    ⎪I ⎪    15⎪(128, 128) ⎪1 MHz CON     ⎪2015-09-04 16:34:43.0 ⎪ 00ʰ00ᵐ02.3ˢ⎪
⎪SHA_20150904.0022 ⎪74in ⎪SKYFLAT         ⎪flat    ⎪∅ ⎪    15⎪(128, 128) ⎪1 MHz CON     ⎪2015-09-04 16:39:23.0 ⎪ 00ʰ00ᵐ02.3ˢ⎪
⎪SHA_20160710.0003 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪   500⎪(128, 128) ⎪1 MHz CON     ⎪2016-07-10 16:06:48.0 ⎪ 00ʰ01ᵐ16.6ˢ⎪
⎪SHA_20150715.0200 ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪  3584⎪(128, 128) ⎪1 MHz CON     ⎪2015-07-16 01:43:03.0 ⎪ 00ʰ59ᵐ44.0ˢ⎪
⎪SHA_20150904.0201 ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪I ⎪  7201⎪(128, 128) ⎪1 MHz CON     ⎪2015-09-04 19:35:00.0 ⎪ 02ʰ00ᵐ01.0ˢ⎪
⎪SHA_20150904.0202 ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪  9001⎪(128, 128) ⎪1 MHz CON     ⎪2015-09-04 22:40:00.0 ⎪ 02ʰ30ᵐ01.0ˢ⎪
⎪SHA_20150905.0101 ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 12600⎪(128, 128) ⎪1 MHz CON     ⎪2015-09-05 17:43:00.0 ⎪ 03ʰ30ᵐ00.0ˢ⎪
⎪                                                                                                                           ⎪
⎪group 3:                                       8x8; 4.9; EM; 1; 65; 0.14862
⎪SHA_20160712.0109 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪  1000⎪(128, 128) ⎪1 MHz EM: 65  ⎪2016-07-13 05:46:45.0 ⎪ 00ʰ02ᵐ35.4ˢ⎪
⎪SHA_20160711.0001 ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 37505⎪(128, 128) ⎪1 MHz EM: 65  ⎪2016-07-11 22:51:50.0 ⎪ 02ʰ40ᵐ29.8ˢ⎪
⎪                                                                                                                           ⎪
⎪group 4:                                      12x12; 4.9; EM; 1; 50; 0.10242
⎪SHA_20160712.0117 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪  1000⎪(85, 85)   ⎪1 MHz EM: 50  ⎪2016-07-13 06:08:52.0 ⎪ 00ʰ01ᵐ49.2ˢ⎪
⎪SHA_20160706.0007 ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 18253⎪(85, 85)   ⎪1 MHz EM: 50  ⎪2016-07-06 23:23:08.0 ⎪ 02ʰ34ᵐ09.9ˢ⎪
⎪SHA_20160707.0030 ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪  4301⎪(85, 85)   ⎪1 MHz EM: 50  ⎪2016-07-07 22:18:55.0 ⎪ 00ʰ36ᵐ19.6ˢ⎪
⎪SHA_20160707.0031 ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 28000⎪(85, 85)   ⎪1 MHz EM: 50  ⎪2016-07-07 22:55:21.0 ⎪ 01ʰ59ᵐ49.3ˢ⎪
⎪                                                                                                                           ⎪
⎪group 5:                                        8x8; 2.4; EM; 3; 50; 0.0566
⎪20130618.0104     ⎪None ⎪None            ⎪dark    ⎪∅ ⎪   100⎪(128, 128) ⎪3 MHz EM: 50  ⎪2013-06-19 02:13:32.0 ⎪ 00ʰ00ᵐ06.3ˢ⎪
⎪20130616.0030     ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 35000⎪(128, 128) ⎪3 MHz EM: 50  ⎪2013-06-16 21:45:55.9 ⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪20130616.0031     ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 35000⎪(128, 128) ⎪3 MHz EM: 50  ⎪2013-06-16 22:48:41.9 ⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪20130617.0030     ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 35000⎪(128, 128) ⎪3 MHz EM: 50  ⎪2013-06-17 21:49:58.9 ⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪20130617.0031     ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 35000⎪(128, 128) ⎪3 MHz EM: 50  ⎪2013-06-17 22:52:57.9 ⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪                                                                                                                           ⎪
⎪group 6:                                       8x8; 2.4; EM; 3; 125; 0.0566
⎪20130618.0105     ⎪None ⎪None            ⎪dark    ⎪∅ ⎪   100⎪(128, 128) ⎪3 MHz EM: 125 ⎪2013-06-19 02:14:01.0 ⎪ 00ʰ00ᵐ06.3ˢ⎪
⎪20130618.0030     ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 35000⎪(128, 128) ⎪3 MHz EM: 125 ⎪2013-06-18 22:41:04.9 ⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪20130618.0031     ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 22999⎪(128, 128) ⎪3 MHz EM: 125 ⎪2013-06-18 23:43:58.9 ⎪ 00ʰ40ᵐ55.4ˢ⎪
⎪                                                                                                                           ⎪
⎪group 7:                                       12x12; 2.4; CON; 1; 0; 0.1002
⎪SHA_20160712.0116 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪  1000⎪(85, 85)   ⎪1 MHz CON     ⎪2016-07-13 06:06:08.0 ⎪ 00ʰ01ᵐ47.0ˢ⎪
⎪SHA_20160706.0004 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪   405⎪(85, 85)   ⎪1 MHz CON     ⎪2016-07-06 16:14:41.0 ⎪ 00ʰ02ᵐ04.2ˢ⎪
⎪                                                                                                                           ⎪
⎪group 8:                                        8x8; 1.0; CON; 3; 0; 0.0558
⎪SHA_20150607.0010 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪    30⎪(128, 128) ⎪3 MHz CON     ⎪2015-06-07 16:01:12.0 ⎪ 00ʰ00ᵐ01.9ˢ⎪
⎪SHA_20150607.0011 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪    30⎪(128, 128) ⎪3 MHz CON     ⎪2015-06-07 16:01:21.0 ⎪ 00ʰ00ᵐ01.9ˢ⎪
⎪SHA_20150607.0012 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪    30⎪(128, 128) ⎪3 MHz CON     ⎪2015-06-07 16:01:38.0 ⎪ 00ʰ00ᵐ01.9ˢ⎪
⎪SHA_20150606.0008 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪    10⎪(128, 128) ⎪3 MHz CON     ⎪2015-06-06 16:20:40.0 ⎪ 00ʰ01ᵐ40.1ˢ⎪
⎪SHA_20150607.0003 ⎪74in ⎪                ⎪flat    ⎪∅ ⎪    30⎪(128, 128) ⎪3 MHz CON     ⎪2015-06-07 15:57:58.0 ⎪ 00ʰ00ᵐ01.9ˢ⎪
⎪SHA_20150606.0300 ⎪74in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪  4824⎪(128, 128) ⎪3 MHz CON     ⎪2015-06-07 01:25:01.0 ⎪ 00ʰ40ᵐ12.0ˢ⎪
⎪                                                                                                                           ⎪
⎪group 9:                                       8x8; 2.4; CON; 1; 0; 0.14642
⎪SHA_20160712.0105 ⎪74in ⎪                ⎪dark    ⎪∅ ⎪  1000⎪(128, 128) ⎪1 MHz CON     ⎪2016-07-13 05:33:59.0 ⎪ 00ʰ02ᵐ33.2ˢ⎪
⎪20130616.0002     ⎪74in ⎪None            ⎪flat    ⎪∅ ⎪    50⎪(128, 128) ⎪1 MHz CON     ⎪2013-06-16 16:02:16.9 ⎪ 00ʰ00ᵐ07.7ˢ⎪
⎪20130617.0001     ⎪74in ⎪None            ⎪flat    ⎪∅ ⎪   100⎪(128, 128) ⎪1 MHz CON     ⎪2013-06-17 16:02:29.9 ⎪ 00ʰ00ᵐ15.3ˢ⎪
⎪202130615.0002    ⎪74in ⎪None            ⎪flat    ⎪∅ ⎪    50⎪(128, 128) ⎪1 MHz CON     ⎪2013-06-15 16:05:53.9 ⎪ 00ʰ00ᵐ07.7ˢ⎪
⎪SHA_20171002.0010 ⎪40in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪   670⎪(128, 128) ⎪1 MHz CON     ⎪2017-10-02 18:35:38.0 ⎪ 00ʰ05ᵐ39.5ˢ⎪
⎪SHA_20171002.0011 ⎪40in ⎪CTCV J1928-5001 ⎪object  ⎪∅ ⎪ 12498⎪(128, 128) ⎪1 MHz CON     ⎪2017-10-02 18:59:44.0 ⎪ 01ʰ45ᵐ33.5ˢ⎪
* t0: timing.trigger.is_gps
* tExp: timing.trigger.is_gps_loop

In [ ]:
# Notice that i have previously placed all the correct dark and flat 
# observation files in the one folder. This is actually not necessary. If 
# you already have a folder where you store all your calibration data, you can
# read all those files and select the correct ones with a few lines of code 
# similar to what was done above. 
# For example:
# darks = shocCampaign.load_dir('/media/Oceanus/UCT/Observing/data/darks')
#_ = run_src.match(darks, *ATT_DARK, keep_nulls=False, report=True)
In [10]:
# display
fig, _ = mbias.to_list().sort_by('date').thumbnails(title=('date', 'readout'))
fig
Out[10]:
In [355]:
from obstools.stats import median_scaled_median

# Science files are large and won't all fit into RAM, so needs careful handling.
# use `set_calibrators` to do calibration arithmetic on the fly when accessing 
# data via `calibrated` attribute
gobj = gobs.select_by(obstype='object')
gobj.set_calibrators(mbias)

# Flat fields are small enough volume that we can safely read them into RAM.
gflat = gobs.select_by(obstype='flat')
gflat = gflat.subtract(mbias)

# Match calibrated flat fields to science observations. This grouping will be 
# different to that of `gflat` above since we are now matching for closest 
# dates
ATT_EQUAL_FLAT = ('telescope', 'instrument', 'binning', 'filters')
ATT_CLOSE_FLAT = ('date', )
ATT_FLAT = ATT_EQUAL_FLAT, ATT_CLOSE_FLAT

gobj, gflat = g['object'].match(gflat.to_list(), *ATT_FLAT, keep_nulls=False,
                                report=True)

# flat field: median scale each image, then median combine images
mflat = gflat.merge_combine(median_scaled_median)
# 
gobj.set_calibrators(flats=mflat)
                                                                                                                                   
⎪Matched Observations                                                                                                             ⎪
⎪exact: ('telescope', 'instrument', 'binning', 'filters')                                                                         ⎪
⎪closest: ('date',)                                                                                                               ⎪
⎪None                                                                                                                             ⎪
⎪                                                               ⎪      readout       ⎪                   timing                   ⎪
⎪     filename     ⎪     target     ⎪obstype ⎪  n   ⎪  ishape   ⎪γₚᵣₑ ⎪     mode     ⎪          t0          ⎪  tExp  ⎪  duration  ⎪
⎪                                                                                                                                 ⎪
⎪group 0:                                    74in; 8x8; Filters('∅', '∅'); 2015-07-15
⎪SHA_20150715.0002 ⎪SKYFLAT         ⎪flat    ⎪    15⎪(128, 128) ⎪  1.0⎪1 MHz CON     ⎪2015-07-15 16:07:32.0 ⎪ 0.14642⎪ 00ʰ00ᵐ02.3ˢ⎪
⎪SHA_20150715.0200 ⎪CTCV J1928-5001 ⎪object  ⎪  3584⎪(128, 128) ⎪  1.0⎪1 MHz CON     ⎪2015-07-16 01:43:03.0 ⎪ 0.99324⎪ 00ʰ59ᵐ44.0ˢ⎪
⎪                                                                                                                                 ⎪
⎪group 1:                                    74in; 8x8; Filters('∅', '∅'); 2015-09-04
⎪SHA_20150904.0022 ⎪SKYFLAT         ⎪flat    ⎪    15⎪(128, 128) ⎪  1.0⎪1 MHz CON     ⎪2015-09-04 16:39:23.0 ⎪ 0.14642⎪ 00ʰ00ᵐ02.3ˢ⎪
⎪SHA_20150904.0202 ⎪CTCV J1928-5001 ⎪object  ⎪  9001⎪(128, 128) ⎪  1.0⎪1 MHz CON     ⎪2015-09-04 22:40:00.0 ⎪ 0.99324⎪ 02ʰ30ᵐ01.0ˢ⎪
⎪SHA_20150905.0101 ⎪CTCV J1928-5001 ⎪object  ⎪ 12600⎪(128, 128) ⎪  1.0⎪1 MHz CON     ⎪2015-09-05 17:43:00.0 ⎪ 0.99324⎪ 03ʰ30ᵐ00.0ˢ⎪
⎪                                                                                                                                 ⎪
⎪group 2:                                    74in; 8x8; Filters('∅', '∅'); 2016-07-10
⎪SHA_20160710.0003 ⎪                ⎪flat    ⎪   500⎪(128, 128) ⎪  1.0⎪1 MHz CON     ⎪2016-07-10 16:06:48.0 ⎪ 0.14642⎪ 00ʰ01ᵐ16.6ˢ⎪
⎪SHA_20160711.0001 ⎪CTCV J1928-5001 ⎪object  ⎪ 37505⎪(128, 128) ⎪  4.9⎪1 MHz EM: 65  ⎪2016-07-11 22:51:50.0 ⎪    0.25⎪ 02ʰ40ᵐ29.8ˢ⎪
⎪                                                                                                                                 ⎪
⎪group 3:                                    74in; 8x8; Filters('∅', '∅'); 2015-06-07
⎪SHA_20150607.0003 ⎪                ⎪flat    ⎪    30⎪(128, 128) ⎪  1.0⎪3 MHz CON     ⎪2015-06-07 15:57:58.0 ⎪  0.0558⎪ 00ʰ00ᵐ01.9ˢ⎪
⎪SHA_20150606.0300 ⎪CTCV J1928-5001 ⎪object  ⎪  4824⎪(128, 128) ⎪  1.0⎪3 MHz CON     ⎪2015-06-07 01:25:01.0 ⎪ 0.49324⎪ 00ʰ40ᵐ12.0ˢ⎪
⎪                                                                                                                                 ⎪
⎪group 4:                                    74in; 8x8; Filters('∅', '∅'); 2013-06-16
⎪20130616.0002     ⎪None            ⎪flat    ⎪    50⎪(128, 128) ⎪  2.4⎪1 MHz CON     ⎪2013-06-16 16:02:16.9 ⎪ 0.14642⎪ 00ʰ00ᵐ07.7ˢ⎪
⎪20130616.0030     ⎪CTCV J1928-5001 ⎪object  ⎪ 35000⎪(128, 128) ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-16 21:45:55.9 ⎪     0.1⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪20130616.0031     ⎪CTCV J1928-5001 ⎪object  ⎪ 35000⎪(128, 128) ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-16 22:48:41.9 ⎪     0.1⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪202130615.0031    ⎪CTCV J1928-5001 ⎪object  ⎪ 20000⎪(128, 128) ⎪  2.4⎪1 MHz EM: 50  ⎪2013-06-16 00:07:07.9 ⎪ 0.14862⎪ 00ʰ51ᵐ47.6ˢ⎪
⎪202130615.0032    ⎪CTCV J1928-5001 ⎪object  ⎪ 10000⎪(128, 128) ⎪  2.4⎪1 MHz EM: 50  ⎪2013-06-16 01:01:08.9 ⎪ 0.14862⎪ 00ʰ25ᵐ53.8ˢ⎪
⎪                                                                                                                                 ⎪
⎪group 5:                                    74in; 8x8; Filters('∅', '∅'); 2013-06-17
⎪20130617.0001     ⎪None            ⎪flat    ⎪   100⎪(128, 128) ⎪  2.4⎪1 MHz CON     ⎪2013-06-17 16:02:29.9 ⎪ 0.14642⎪ 00ʰ00ᵐ15.3ˢ⎪
⎪20130617.0030     ⎪CTCV J1928-5001 ⎪object  ⎪ 35000⎪(128, 128) ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-17 21:49:58.9 ⎪     0.1⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪20130617.0031     ⎪CTCV J1928-5001 ⎪object  ⎪ 35000⎪(128, 128) ⎪  2.4⎪3 MHz EM: 50  ⎪2013-06-17 22:52:57.9 ⎪     0.1⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪20130618.0030     ⎪CTCV J1928-5001 ⎪object  ⎪ 35000⎪(128, 128) ⎪  2.4⎪3 MHz EM: 125 ⎪2013-06-18 22:41:04.9 ⎪     0.1⎪ 01ʰ02ᵐ16.6ˢ⎪
⎪20130618.0031     ⎪CTCV J1928-5001 ⎪object  ⎪ 22999⎪(128, 128) ⎪  2.4⎪3 MHz EM: 125 ⎪2013-06-18 23:43:58.9 ⎪     0.1⎪ 00ʰ40ᵐ55.4ˢ⎪
⎪                                                                                                                                 ⎪
⎪group 6:                                    74in; 8x8; Filters('∅', '∅'); 2013-06-15
⎪202130615.0002    ⎪None            ⎪flat    ⎪    50⎪(128, 128) ⎪  2.4⎪1 MHz CON     ⎪2013-06-15 16:05:53.9 ⎪ 0.14642⎪ 00ʰ00ᵐ07.7ˢ⎪
⎪202130615.0030    ⎪CTCV J1928-5001 ⎪object  ⎪ 20000⎪(128, 128) ⎪  2.4⎪1 MHz EM: 50  ⎪2013-06-15 23:15:00.9 ⎪ 0.14862⎪ 00ʰ51ᵐ47.6ˢ⎪
⎪                                                                                                                                 ⎪
⎪group 7:                                          40in; 8x8; Filters('∅', '∅')NO MATCH          ⎪                ⎪        ⎪      ⎪           ⎪     ⎪              ⎪                      ⎪        ⎪            ⎪
⎪SHA_20171002.0010 ⎪CTCV J1928-5001 ⎪object  ⎪   670⎪(128, 128) ⎪  2.4⎪1 MHz CON     ⎪2017-10-02 18:35:38.0 ⎪     0.5⎪ 00ʰ05ᵐ39.5ˢ⎪
⎪SHA_20171002.0011 ⎪CTCV J1928-5001 ⎪object  ⎪ 12498⎪(128, 128) ⎪  2.4⎪1 MHz CON     ⎪2017-10-02 18:59:44.0 ⎪     0.5⎪ 01ʰ45ᵐ33.5ˢ⎪
⎪                                                                                                                                 ⎪
⎪group 8:                                   74in; 12x12; Filters('∅', '∅'); 2016-07-07
⎪SHA_20160707.0004 ⎪                ⎪flat    ⎪   633⎪(85, 85)   ⎪  1.0⎪1 MHz CON     ⎪2016-07-07 16:11:08.0 ⎪ 0.11847⎪ 00ʰ01ᵐ19.3ˢ⎪
⎪SHA_20160707.0030 ⎪CTCV J1928-5001 ⎪object  ⎪  4301⎪(85, 85)   ⎪  4.9⎪1 MHz EM: 50  ⎪2016-07-07 22:18:55.0 ⎪     0.5⎪ 00ʰ36ᵐ19.6ˢ⎪
⎪SHA_20160707.0031 ⎪CTCV J1928-5001 ⎪object  ⎪ 28000⎪(85, 85)   ⎪  4.9⎪1 MHz EM: 50  ⎪2016-07-07 22:55:21.0 ⎪    0.25⎪ 01ʰ59ᵐ49.3ˢ⎪
⎪                                                                                                                                 ⎪
⎪group 9:                                   74in; 12x12; Filters('∅', '∅'); 2016-07-06
⎪SHA_20160706.0004 ⎪                ⎪flat    ⎪   405⎪(85, 85)   ⎪  2.4⎪1 MHz CON     ⎪2016-07-06 16:14:41.0 ⎪     0.3⎪ 00ʰ02ᵐ04.2ˢ⎪
⎪SHA_20160706.0007 ⎪CTCV J1928-5001 ⎪object  ⎪ 18253⎪(85, 85)   ⎪  4.9⎪1 MHz EM: 50  ⎪2016-07-06 23:23:08.0 ⎪     0.5⎪ 02ʰ34ᵐ09.9ˢ⎪
⎪                                                                                                                                 ⎪
⎪group 10:                                   74in; 8x8; Filters('I', '∅'); 2015-09-04
⎪SHA_20150904.0012 ⎪SKYFLAT         ⎪flat    ⎪    15⎪(128, 128) ⎪  1.0⎪1 MHz CON     ⎪2015-09-04 16:34:43.0 ⎪ 0.14642⎪ 00ʰ00ᵐ02.3ˢ⎪
⎪SHA_20150904.0201 ⎪CTCV J1928-5001 ⎪object  ⎪  7201⎪(128, 128) ⎪  1.0⎪1 MHz CON     ⎪2015-09-04 19:35:00.0 ⎪ 0.99324⎪ 02ʰ00ᵐ01.0ˢ⎪
* t0: timing.trigger.is_gps
* tExp: timing.trigger.is_gps_loop

In [26]:
# display master flats
fig, _ = mflat.to_list().sort_by('date').thumbnails(title='date')
fig
Out[26]:
In [7]:
# show image grid (calibrated images)

# gobj.set_calibrators(mbias) # mflat
orun =  gobj.to_list().sort_by('date')
fig, _ = orun.thumbnails(calibrated=True)
fig
Out[7]:
In [356]:
# The flats for 20150606.0300.fits seem to introduce artifacts rather than 
# remove them.  This is most likely due to small number statistics since 
# there are only 30 frames in the flat field observation..
orun =  gobj.to_list().sort_by('date')
orun[9].calibrated.flat = None

# Also don't have calibration images for 1.0m data, which can't be helped
In [358]:
# Image registration & mosaic

import cmasher as cmr

# align images
reg = orun.coalign(plot=False) 

# plot
mp = reg.mosaic(alpha=0.35, cmap='cmr.dusk', number_sources=True)
mp.fig
INFO:core.shocOldHDU:Computing median of 67 images (exposure depth of 10.0 seconds) for sample image from '202130615.0030.fits'
INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130616.0030.fits'
INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130616.0031.fits'
INFO:core.shocOldHDU:Computing median of 67 images (exposure depth of 10.0 seconds) for sample image from '202130615.0031.fits'
INFO:core.shocOldHDU:Computing median of 67 images (exposure depth of 10.0 seconds) for sample image from '202130615.0032.fits'
INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130617.0030.fits'
INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130617.0031.fits'
INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130618.0030.fits'
INFO:core.shocOldHDU:Computing median of 99 images (exposure depth of 10.0 seconds) for sample image from '20130618.0031.fits'
INFO:core.shocNewHDU:Computing median of 20 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20150606.0300.fits'
INFO:core.shocNewHDU:Computing median of 10 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20150715.0200.fits'
INFO:core.shocNewHDU:Computing median of 10 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20150904.0202.fits'
INFO:core.shocNewHDU:Computing median of 10 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20150904.0201.fits'
INFO:core.shocNewHDU:Computing median of 10 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20150905.0101.fits'
INFO:core.shocFlatHDU:Computing median of 20 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20160706.0007.fits'
INFO:core.shocNewHDU:Computing median of 20 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20160707.0030.fits'
INFO:core.shocNewHDU:Computing median of 40 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20160707.0031.fits'
INFO:core.shocNewHDU:Computing median of 40 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20160711.0001.fits'
INFO:registration.ImageRegister:Aligning 18 images on image 0
INFO:image.registration:Identifying stars
INFO:image.registration:Clustering 189 position measurements using:
	MeanShift(bandwidth=5.94703255887994, bin_seeding=False, cluster_all=False,
	          min_bin_freq=1, n_jobs=None, seeds=None)
INFO:image.registration:Identified 23 stars using 189/189 points (0 noise)
INFO:image.registration:Identifying stars
INFO:image.registration:Clustering 189 position measurements using:
	MeanShift(bandwidth=5.94703255887994, bin_seeding=False, cluster_all=False,
	          min_bin_freq=1, n_jobs=None, seeds=None)
INFO:image.registration:Identified 23 stars using 189/189 points (0 noise)
INFO:registration.ImageRegister:Fitting successful 17 / 17
INFO:registration.ImageRegister:Likelihood ratio: 0.99873
INFO:registration.ImageRegister:Keeping same parameters.
INFO:image.registration:Measuring cluster centres, frame xy-offsets
INFO:image.registration:Ignoring 8 / 23 stars with low (<=25%) detection frequency for frame offset measurement.
INFO:image.registration:
                                                  
⎪            Measured star locations             ⎪
⎪#  ⎪       n (%)⎪              x⎪              y⎪
⎪1  ⎪   18 (100%)⎪   55.74 ± 0.09⎪   92.01 ± 0.18⎪
⎪2  ⎪   18 (100%)⎪   47.37 ± 0.16⎪   41.55 ± 0.13⎪
⎪3  ⎪   18 (100%)⎪   72.54 ± 0.13⎪   38.54 ± 0.21⎪
⎪4  ⎪   18 (100%)⎪   57.74 ± 0.15⎪   28.44 ± 0.19⎪
⎪5  ⎪    14 (78%)⎪   15.83 ± 0.07⎪   96.26 ± 0.11⎪
⎪6  ⎪    13 (72%)⎪   93.01 ± 0.11⎪   43.19 ± 0.13⎪
⎪7  ⎪    12 (67%)⎪   27.42 ± 0.52⎪   49.33 ± 0.91⎪
⎪8  ⎪    11 (61%)⎪  108.03 ± 0.13⎪   74.48 ± 0.21⎪
⎪9  ⎪     8 (44%)⎪    2.80 ± 0.24⎪   86.53 ± 0.20⎪
⎪10 ⎪     8 (44%)⎪   -4.50 ± 0.12⎪   29.69 ± 0.28⎪
⎪11 ⎪     7 (39%)⎪   64.59 ± 0.59⎪  124.95 ± 0.30⎪
⎪12 ⎪     7 (39%)⎪   -5.45 ± 0.20⎪  122.24 ± 0.21⎪
⎪13 ⎪     7 (39%)⎪  -12.13 ± 0.14⎪   41.26 ± 0.24⎪
⎪14 ⎪     5 (28%)⎪    5.44 ± 0.43⎪  128.03 ± 0.29⎪
⎪15 ⎪     5 (28%)⎪  100.83 ± 0.18⎪   23.85 ± 0.39⎪
⎪16 ⎪     4 (22%)⎪ -28.74 ± 51.63⎪  74.79 ± 51.63⎪
⎪17 ⎪     3 (17%)⎪  77.75 ± 16.80⎪ 111.20 ± 16.80⎪
⎪18 ⎪     3 (17%)⎪ 129.30 ± 33.93⎪  61.32 ± 33.93⎪
⎪19 ⎪     3 (17%)⎪ 132.96 ± 67.83⎪  -2.68 ± 67.83⎪
⎪20 ⎪     2 (11%)⎪   -7.93 ± 5.80⎪    3.64 ± 5.80⎪
⎪21 ⎪     2 (11%)⎪ 115.71 ± 61.59⎪  -7.47 ± 61.59⎪
⎪22 ⎪     2 (11%)⎪ 143.47 ± 80.92⎪ -18.37 ± 80.92⎪
⎪23 ⎪      1 (6%)⎪ 120.39 ± 71.78⎪ -23.16 ± 71.78⎪
⎪   ⎪ 189 (1050%)⎪               ⎪               ⎪
n_noise = 225/414 (54.3%)
INFO:registration.ImageRegister:Likelihood ratio: nan
INFO:registration.ImageRegister:Keeping same parameters.
INFO:core.shocNewHDU:Computing median of 20 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20171002.0010.fits'
INFO:core.shocNewHDU:Computing median of 20 images (exposure depth of 10.0 seconds) for sample image from 'SHA_20171002.0011.fits'
INFO:registration.ImageRegister:Aligning 2 images on image 0
INFO:image.registration:Identifying stars
INFO:image.registration:Clustering 21 position measurements using:
	MeanShift(bandwidth=2.343487033951146, bin_seeding=False, cluster_all=False,
	          min_bin_freq=1, n_jobs=None, seeds=None)
INFO:image.registration:Identified 13 stars using 21/21 points (0 noise)
INFO:image.registration:Identifying stars
INFO:image.registration:Clustering 21 position measurements using:
	MeanShift(bandwidth=2.343487033951146, bin_seeding=False, cluster_all=False,
	          min_bin_freq=1, n_jobs=None, seeds=None)
INFO:image.registration:Identified 13 stars using 21/21 points (0 noise)
INFO:registration.ImageRegister:Fitting successful 1 / 1
INFO:registration.ImageRegister:Likelihood ratio: 0.99916
INFO:registration.ImageRegister:Keeping same parameters.
INFO:image.registration:Measuring cluster centres, frame xy-offsets
INFO:image.registration:
                                              
⎪          Measured star locations           ⎪
⎪#  ⎪      n (%)⎪            x⎪             y⎪
⎪1  ⎪   2 (100%)⎪ 57.56 ± 0.02⎪ 116.71 ± 0.01⎪
⎪2  ⎪   2 (100%)⎪ 53.74 ± 0.01⎪  93.79 ± 0.03⎪
⎪3  ⎪   2 (100%)⎪ 19.11 ± 0.07⎪  67.73 ± 0.05⎪
⎪4  ⎪   2 (100%)⎪ 97.06 ± 0.04⎪  66.52 ± 0.07⎪
⎪5  ⎪   2 (100%)⎪ 46.95 ± 0.06⎪  41.71 ± 0.07⎪
⎪6  ⎪   2 (100%)⎪ 80.76 ± 0.09⎪  38.67 ± 0.03⎪
⎪7  ⎪   2 (100%)⎪ 81.93 ± 0.09⎪  29.36 ± 0.03⎪
⎪8  ⎪   2 (100%)⎪  6.56 ± 0.04⎪  26.95 ± 0.12⎪
⎪9  ⎪    1 (50%)⎪ -4.13 ± 0.00⎪  99.71 ± 0.00⎪
⎪10 ⎪    1 (50%)⎪ 30.60 ± 0.00⎪  88.42 ± 0.00⎪
⎪11 ⎪    1 (50%)⎪ 59.49 ± 0.00⎪  62.37 ± 0.00⎪
⎪12 ⎪    1 (50%)⎪ 54.90 ± 0.00⎪  43.18 ± 0.00⎪
⎪13 ⎪    1 (50%)⎪  1.85 ± 0.00⎪  26.52 ± 0.00⎪
⎪   ⎪ 21 (1050%)⎪             ⎪              ⎪
n_noise = 5/26 (19.2%)
INFO:registration.ImageRegister:Likelihood ratio: 0.99942
INFO:registration.ImageRegister:Keeping same parameters.
WARNING:registration.CoherentPointDrift:CoherentPointDrift optimization with objective 'Model.loss_mle' failed to converge: Desired error not necessarily achieved due to precision loss.
INFO:image.registration:Identifying stars
INFO:image.registration:Clustering 210 position measurements using:
	MeanShift(bandwidth=5.177471354078108, bin_seeding=False, cluster_all=False,
	          min_bin_freq=1, n_jobs=None, seeds=None)
INFO:image.registration:Identified 32 stars using 210/210 points (0 noise)
INFO:registration.ImageRegister:Fitting successful 19 / 19
INFO:registration.ImageRegister:Likelihood ratio: 0.99881
INFO:registration.ImageRegister:Keeping same parameters.
INFO:image.registration:Measuring cluster centres, frame xy-offsets
INFO:image.registration:Ignoring 19 / 32 stars with low (<=25%) detection frequency for frame offset measurement.
INFO:image.registration:
                                                     
⎪              Measured star locations              ⎪
⎪#  ⎪       n (%)⎪               x⎪                y⎪
⎪1  ⎪   20 (100%)⎪    55.73 ± 0.11⎪     92.04 ± 0.17⎪
⎪2  ⎪   20 (100%)⎪    47.35 ± 0.15⎪     41.53 ± 0.11⎪
⎪3  ⎪    18 (90%)⎪    72.54 ± 0.13⎪     38.53 ± 0.22⎪
⎪4  ⎪    18 (90%)⎪    57.74 ± 0.14⎪     28.44 ± 0.19⎪
⎪5  ⎪    14 (70%)⎪    15.85 ± 0.08⎪     96.25 ± 0.11⎪
⎪6  ⎪    13 (65%)⎪    93.01 ± 0.10⎪     43.19 ± 0.15⎪
⎪7  ⎪    12 (60%)⎪    27.42 ± 0.50⎪     49.32 ± 0.90⎪
⎪8  ⎪    11 (55%)⎪   108.03 ± 0.12⎪     74.47 ± 0.19⎪
⎪9  ⎪     9 (45%)⎪    -4.49 ± 0.19⎪     29.68 ± 0.26⎪
⎪10 ⎪     8 (40%)⎪     2.80 ± 0.23⎪     86.54 ± 0.20⎪
⎪11 ⎪     7 (35%)⎪    64.59 ± 0.62⎪    124.95 ± 0.31⎪
⎪12 ⎪     7 (35%)⎪    -5.44 ± 0.23⎪    122.23 ± 0.22⎪
⎪13 ⎪     7 (35%)⎪   -12.13 ± 0.12⎪     41.26 ± 0.25⎪
⎪14 ⎪     5 (25%)⎪    5.45 ± 61.37⎪   128.03 ± 61.37⎪
⎪15 ⎪     5 (25%)⎪  100.83 ± 38.56⎪    23.85 ± 38.56⎪
⎪16 ⎪     4 (20%)⎪  -28.74 ± 51.64⎪    74.79 ± 51.64⎪
⎪17 ⎪     4 (20%)⎪  143.39 ± 80.88⎪   -18.42 ± 80.88⎪
⎪18 ⎪     3 (15%)⎪   77.79 ± 16.83⎪   111.21 ± 16.83⎪
⎪19 ⎪     3 (15%)⎪  129.30 ± 33.93⎪    61.31 ± 33.93⎪
⎪20 ⎪     3 (15%)⎪  132.96 ± 67.83⎪    -2.68 ± 67.83⎪
⎪21 ⎪     2 (10%)⎪    -7.93 ± 5.80⎪      3.65 ± 5.80⎪
⎪22 ⎪     2 (10%)⎪  115.70 ± 61.59⎪    -7.48 ± 61.59⎪
⎪23 ⎪     2 (10%)⎪   -29.29 ± 6.61⎪    -16.07 ± 6.61⎪
⎪24 ⎪     2 (10%)⎪   32.22 ± 52.88⎪   -73.55 ± 52.88⎪
⎪25 ⎪     2 (10%)⎪  106.91 ± 93.59⎪   -80.27 ± 93.59⎪
⎪26 ⎪     2 (10%)⎪ 109.50 ± 105.18⎪ -100.85 ± 105.18⎪
⎪27 ⎪     2 (10%)⎪  -57.01 ± 24.58⎪  -106.17 ± 24.58⎪
⎪28 ⎪      1 (5%)⎪  -80.72 ± 67.64⎪    54.57 ± 67.64⎪
⎪29 ⎪      1 (5%)⎪  120.37 ± 71.78⎪   -23.19 ± 71.78⎪
⎪30 ⎪      1 (5%)⎪   60.00 ± 43.94⎪   -27.89 ± 43.94⎪
⎪31 ⎪      1 (5%)⎪   49.87 ± 60.09⎪   -70.30 ± 60.09⎪
⎪32 ⎪      1 (5%)⎪  -67.52 ± 19.80⎪  -107.13 ± 19.80⎪
⎪   ⎪ 210 (1050%)⎪                ⎪                 ⎪
n_noise = 430/640 (67.2%)
INFO:registration.ImageRegister:Likelihood ratio: 0.99974
INFO:registration.ImageRegister:Keeping same parameters.
Out[358]:
In [359]:
from graphing.imagine import ImageDisplay

def pixel_transform(i: int):
    # scale images by source counts for source 0 and median subtract
    image = reg[i]
    a = image.data / image.counts[sidx[i] == 0]
    return a - np.ma.median(a)

# get source indices per frame
sidx = reg.source_indices
# get mean image across stack
g, bs = reg.binned_statistic(image_func=pixel_transform, interpolate=True)

im = ImageDisplay(bs.statistic.T)
im.imagePlot.set_clim(0, 0.012)
im.figure.set_size_inches(8, 8)
im.figure
Out[359]: